# -*- coding: utf-8 -*-

#
# peerscape.http.controllers.__init__
#
# Copyright 2008 Helsinki Institute for Information Technology
# and the authors.
#
# Authors:
#       Konrad Markus <konrad.markus@hiit.fi>
#

# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import cgi
import sys
import os
import re
import rfc822
import mimetypes
import gettext
import xml.sax.saxutils
from binascii import hexlify, unhexlify

from peerscape import VERSION
from peerscape.db.invitation import decode_invitation
from peerscape.db.util import get_timestamp
from peerscape.lib import json
from peerscape.lib import utils
from mako.lookup import TemplateLookup

class BaseController(object):
    # basically ../../i18n relative to the controllers directory
    BASE_LC_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), os.path.join('..', '..', 'i18n')))

    # basically ../templates relative to the controllers directory
    BASE_TEMPLATE_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), os.path.join('..', 'templates')))


    def __init__(self):
        self.lookup = TemplateLookup(directories=[BaseController.BASE_TEMPLATE_DIR], filesystem_checks=True, input_encoding='utf-8', output_encoding='utf-8', encoding_errors='replace', default_filters=['decode.utf8', 'h'])

        self._transient_invitations = {}
        self._invitations_awake = False

    def load_view(self, view_name):
        return self.lookup.get_template(view_name)
                    
    def _get_mime_type(self, filename):
        if filename.endswith('.xpi'):
            return 'application/x-xpinstall'
        type, encoding = mimetypes.guess_type(filename)
        return type or 'text/plain'
                    
    
    def _get_stat(self, path):
        mtime = os.stat(path).st_mtime
        size = os.stat(path).st_size

        return str(mtime), rfc822.formatdate(mtime), self._get_mime_type(path), str(size)


    def init_translation(self, db):
        try:
            iid = db.get_my_iid()
            pi = db.get_personal_info(iid)
            lang, public = pi.get_field('location', 'lang')
        except:
            lang = 'en'

        if not lang:
            # [FIXME: hardcoded?]
            lang = 'en'

        trans = gettext.translation('http', BaseController.BASE_LC_DIR, [lang])
        trans.install()


    def get_global_props(self, db):
        # common setup for all templates
        self.init_translation(db)

        v = {}
        v['my_iid'] = db.get_my_iid()
        v['my_name'] = db.get_profile_name(v['my_iid']) or _('You')

        return v

    def require_type(self, req, types):
        '''require types and if does not exist return redirect url to index. types is a list of strings.'''
        if req.urlvars.has_key('peerscape.hexlabel'):
            iid = req.urlvars.get('peerscape.hexlabel')
            if iid:
                db = req.environ['peerscape.db']
                actual_type = db.blocking_get_dataset_type(iid)
                if not actual_type in types:
                    return str('http://%s.hexlabel.net/' % req.urlvars['peerscape.hexlabel'])

        return None

    def exclude_type(self, req, types):
        '''exclude given types, returns redirect url to index. types is a list of strings.'''
        if req.urlvars.has_key('peerscape.hexlabel'):
            iid = req.urlvars.get('peerscape.hexlabel')
            if iid:
                db = req.environ['peerscape.db']
                actual_type = db.blocking_get_dataset_type(iid)
                if actual_type in types:
                    return str('http://%s.hexlabel.net/' % req.urlvars['peerscape.hexlabel'])

        return None
        

    def _sanitize(self, s):
        '''esacpes html entities'''
        return cgi.escape(s)

    # NOTE: most of the 'process_...' methods require
    # init_translation() to be called before they are invoked.
    def process_friend_list(self, db, iid, my_iid='', name_sort=True):
        v = {}
        v['index'] = []
        v['items'] = {}
        v['unverified'] = 0

        friend_list = db.list_friends(iid)

        my_friend_list = []
        if not iid == my_iid:
            my_friend_list = db.list_friends(my_iid)

        for fiid in friend_list:
            v['index'].append(fiid)

            v['items'][fiid] = {}
            v['items'][fiid]['iid'] = fiid
            v['items'][fiid]['stable'] = db.get(fiid, '/info/modified')
            v['items'][fiid]['name'] = db.get_profile_name(fiid)
            if not v['items'][fiid]['name']:
                v['items'][fiid]['name'] = db.get_json(iid, '/auth/'+fiid)
                if v['items'][fiid]['name'] and 'name' in v['items'][fiid]['name']:
                    v['items'][fiid]['name'] = v['items'][fiid]['name']['name']
                else:
                    v['items'][fiid]['name'] = _('Unknown Person')
                    
            v['items'][fiid]['picture'] = db.get_profile_picture(fiid)
            if not v['items'][fiid]['picture']:
                v['items'][fiid]['picture'] = '/static/dummy_person.png'
            else:
                v['items'][fiid]['picture'] = 'http://%s.hexlabel.net/get%s' % (fiid, v['items'][fiid]['picture'])
            
            if fiid == my_iid:
                v['items'][fiid]['is_self'] = True
            else:
                v['items'][fiid]['is_self'] = False
            
            if fiid in my_friend_list:
                v['items'][fiid]['is_mutual'] = True
            else:
                v['items'][fiid]['is_mutual'] = False

        if name_sort:
            # sort by name
            v['index'].sort(key=lambda x: v['items'][x]['name'].lower())
        else:
            v['index'].reverse()

        return v


    def process_atom_list(self, db, iid, show_group_content=True, limit=-1):
        v = {'items':[], 'total':0}

        # get atom feeds from links (content/groups)
        links = db.list_links(iid)
        for l in links:
            if l['type'] == 'content':
                # try to get the atom feed
                atom = db.get_json(l['iid'], '/content/feeds/atom.json')
                if atom:
                    atom['parent'] = {}
                    atom['parent']['iid'] = iid
                    atom['parent']['name'] = db.get_profile_name(iid)
                    if not atom['parent']['name']:
                        atom['parent']['name'] = _('Unkown')

                    # check is is a 'redirect' atom feed,
                    # i.e. pointing to a directory where entry.json items can be found
                    if atom.get('peerscape_redir'):
                        entries = []
                        path = atom['peerscape_redir']
                        sub_paths = db.list(l['iid'], path)
                        for p in sub_paths:
                            if path.endswith('/'):
                                full_path = path+p
                            else:
                                full_path = path+'/'+p

                            if full_path.endswith('/entry.json'):
                                value = db.get(l['iid'], full_path)
                                if value:
                                    entries.append(json.loads(value.decode('utf-8')))

                        # sort the entries by updated
                        entries.sort(key=lambda x: x['updated'])
                        atom['entries'] = entries
                        atom['entries'].reverse()

                    # limit entries
                    atom['entries'] = atom['entries'][0:2]

                    # unescape content
                    for e in atom['entries']:
                        e['content']['value'] = xml.sax.saxutils.unescape(e['content']['value'].replace('&quot;', '"'))
                        e['pretty_updated'] = utils.prettydate(e['updated'])

                        # convert urls to links
                        e['summary'] = re.sub(r'(http:\/\/[^\s]+)', r'<a href="\1">\1</a>', e['summary'])

                    if atom['entries']:
                        v['items'].append(atom)

            elif l['type'] == 'group' and show_group_content:
                # only get group content feeds if viewing own profile
                if iid == db.get_my_iid():
                    glinks = db.list_links(l['iid'])
                    for gl in glinks:
                        if gl['type'] == 'content':
                            # try to get the atom feed
                            gatom = db.get_json(gl['iid'], '/content/feeds/atom.json')
                            if gatom:
                                gatom['parent'] = {}
                                gatom['parent']['iid'] = l['iid']
                                gatom['parent']['name'] = db.get_profile_name(l['iid'])
                                if not gatom['parent']['name']:
                                    gatom['parent']['name'] = _('Unkown')

                                # check is is a 'redirect' atom feed,
                                # i.e. pointing to a directory where entry.json items can be found
                                if gatom.get('peerscape_redir'):
                                    entries = []
                                    path = gatom['peerscape_redir']
                                    sub_paths = db.list(gl['iid'], path)
                                    for p in sub_paths:
                                        if path.endswith('/'):
                                            full_path = path+p
                                        else:
                                            full_path = path+'/'+p

                                        if full_path.endswith('/entry.json'):
                                            value = db.get(gl['iid'], full_path)
                                            if value:
                                                entries.append(json.loads(value))

                                    # sort the entries by updated
                                    entries.sort(key=lambda x: x['updated'])
                                    gatom['entries'] = entries

                                # limit entries
                                gatom['entries'] = gatom['entries'][0:2]

                                # unescape content
                                for e in gatom['entries']:
                                    e['content']['value'] = xml.sax.saxutils.unescape(e['content']['value'].replace('&quot;', '"'))
                                    e['pretty_updated'] = utils.prettydate(e['updated'])

                                    # convert urls to links
                                    e['summary'] = re.sub(r'(http:\/\/[^\s]+)', r'<a href="\1">\1</a>', e['summary'])


                                if gatom['entries']:
                                    v['items'].append(gatom)
         

        # get atom feeds from friends (content) if looking at own profile
        if iid == db.get_my_iid():
            friend_list = db.list_friends(iid)
            for fiid in friend_list:
                links = db.list_links(fiid)
                for l in links:
                    if l['type'] == 'content':
                        # try to get the atom feed
                        atom = db.get_json(l['iid'], '/content/feeds/atom.json')
                        if atom:
                            atom['parent'] = {}
                            atom['parent']['iid'] = fiid
                            atom['parent']['name'] = db.get_profile_name(fiid)
                            if not atom['parent']['name']:
                                atom['parent']['name'] = _('Unkown')

                            # check is is a 'redirect' atom feed,
                            # i.e. pointing to a directory where entry.json items can be found
                            if atom.get('peerscape_redir'):
                                entries = []
                                path = atom['peerscape_redir']
                                sub_paths = db.list(l['iid'], path)
                                for p in sub_paths:
                                    if path.endswith('/'):
                                        full_path = path+p
                                    else:
                                        full_path = path+'/'+p

                                    if full_path.endswith('/entry.json'):
                                        value = db.get(l['iid'], full_path)
                                        if value:
                                            entries.append(json.loads(value.decode('utf-8')))

                                # sort the entries by updated
                                entries.sort(key=lambda x: x['updated'])
                                atom['entries'] = entries
                                atom['entries'].reverse()

                            # limit entries
                            atom['entries'] = atom['entries'][0:2]

                            # unescape content
                            for e in atom['entries']:
                                e['content']['value'] = xml.sax.saxutils.unescape(e['content']['value'].replace('&quot;', '"'))
                                e['pretty_updated'] = utils.prettydate(e['updated'])

                                # convert urls to links
                                e['summary'] = re.sub(r'(http:\/\/[^\s]+)', r'<a href="\1">\1</a>', e['summary'])

                            if atom['entries']:
                                v['items'].append(atom)
        
        v['total'] = len(v['items'])

        # sort the items by updated
        v['items'].sort(key=lambda feed: feed.get('updated'))
        v['items'].reverse()

        # apply limit if given
        if limit >= 0 and limit <= len(v['items']):
            v['items'] = v['items'][0:limit]

        return v


    def process_events_list(self, db, iid, my_iid=''):
        if not my_iid:
            my_iid = iid

        i = 0

        v = {'items':[]}
        e = db.get_last_event(iid)
        
        if e and e.get('comment'):
            v['items'].append({})
            v['items'][i]['event'] = e
            v['items'][i]['comment'] = e['comment']
            v['items'][i]['timestamp'] = e['timestamp']
            v['items'][i]['pretty_timestamp'] = utils.prettydate(e['timestamp'])
            v['items'][i]['source_iid'] = iid
            v['items'][i]['source_stable'] = db.get(iid, '/info/modified')
            v['items'][i]['source_name'] = db.get_profile_name(iid)
            if not v['items'][i]['source_name']:
                v['items'][i]['source_name'] = _('Unknown')
        
            eiid = e.get('iid')
            if eiid:
                v['items'][i]['target_iid'] = eiid
                v['items'][i]['target_stable'] = db.get(eiid, '/info/modified')
                # [FIXME: does event have some other fallback name?]
                v['items'][i]['target_name'] = db.get_profile_name(eiid)
                if not v['items'][i]['target_name']:
                    v['items'][i]['target_name'] = _('Unknown')

            else:
                v['items'][i]['target_iid'] = None
                v['items'][i]['target_stable'] = None
                v['items'][i]['target_name'] = None

            i += 1

        if my_iid == iid:
            # get friends events
            friend_list = db.list_friends(iid)
            for fiid in friend_list:
                e = db.get_last_event(fiid)

                if e and e.get('comment'):
                    v['items'].append({})
                    v['items'][i]['event'] = e
                    v['items'][i]['comment'] = e['comment']
                    v['items'][i]['timestamp'] = utils.utcisostrlocal(e['timestamp'])
                    v['items'][i]['pretty_timestamp'] = utils.prettydate(e['timestamp'])
                    v['items'][i]['source_iid'] = fiid
                    v['items'][i]['source_stable'] = db.get(fiid, '/info/modified')
                    v['items'][i]['source_name'] = db.get_profile_name(fiid)
                    if not v['items'][i]['source_name']:
                        v['items'][i]['source_name'] = db.get_json(iid, '/auth/'+fiid)
                        if v['items'][i]['source_name'] and 'name' in v['items'][i]['source_name']:
                            v['items'][i]['source_name'] = v['items'][i]['source_name']['name']
                        else:
                            v['items'][i]['source_name'] = _('Unknown Person')

                
                    eiid = e.get('iid')
                    if eiid:
                        v['items'][i]['target_iid'] = eiid
                        v['items'][i]['target_stable'] = db.get(eiid, '/info/modified')
                        # [FIXME: does event have some other fallback name?]
                        v['items'][i]['target_name'] = db.get_profile_name(eiid) or _('Unknown')
                    else:
                        v['items'][i]['target_iid'] = None
                        v['items'][i]['target_stable'] = None
                        v['items'][i]['target_name'] = None

                    i += 1

            # get link events
            links = db.list_links(iid)
            for l in links:
                e = db.get_last_event(l['iid'])

                if e and e.get('comment'):
                    v['items'].append({})
                    v['items'][i]['event'] = e
                    v['items'][i]['comment'] = e['comment']
                    v['items'][i]['timestamp'] = utils.utcisostrlocal(e['timestamp'])
                    v['items'][i]['pretty_timestamp'] = utils.prettydate(e['timestamp'])
                    v['items'][i]['source_iid'] = l['iid']
                    v['items'][i]['source_stable'] = db.get(l['iid'], '/info/modified')
                    v['items'][i]['source_name'] = db.get_profile_name(l['iid']) or l['title']
                    if not v['items'][i]['source_name']:
                        v['items'][i]['source_name'] = _('Unknown')
                
                    eiid = e.get('iid')
                    if eiid:
                        v['items'][i]['target_iid'] = eiid
                        v['items'][i]['target_stable'] = db.get(eiid, '/info/modified')
                        # [FIXME: does event have some other fallback name?]
                        v['items'][i]['target_name'] = db.get_profile_name(eiid) or _('Unknown')
                        if not v['items'][i]['target_name']:
                            v['items'][i]['target_name'] = _('Unknown')
                    else:
                        v['items'][i]['target_iid'] = None
                        v['items'][i]['target_stable'] = None
                        v['items'][i]['target_name'] = None

                    i += 1

        # sort by timestamp
        v['items'].sort(key=lambda event: event.get('timestamp'))
        v['items'].reverse()

        return v


    def process_status_list(self, db, iid, limit=-1):
        my_iid = db.get_my_iid()
        v = {'items':[]}
        
        if iid == my_iid:
            i = 0

            friend_list = db.list_friends(iid)
            for fiid in friend_list:
                info = db.get_personal_info(fiid)
                status, public = info.get_field('other', 'status')
                status_timestamp, public = info.get_field('other', 'status_timestamp')

                if status:
                    v['items'].append({})
                    v['items'][i]['comment'] = status
                    v['items'][i]['timestamp'] = status_timestamp
                    v['items'][i]['pretty_timestamp'] = utils.prettydate(status_timestamp)
                    v['items'][i]['source_iid'] = fiid
                    v['items'][i]['source_stable'] = db.get(fiid, '/info/modified')
                    v['items'][i]['source_name'] = db.get_profile_name(fiid)
                    if not v['items'][i]['source_name']:
                        v['items'][i]['source_name'] = _('Unknown')
                
                    i += 1

        # sort by timestamp
        v['items'].sort(key=lambda event: event.get('timestamp'))
        v['items'].reverse()

        # apply limit if given
        if limit >= 0 and limit <= len(v['items']):
            v['items'] = v['items'][0:limit]

        return v


    def get_group_link_visibility(self, db, iid):
        my_iid = db.get_my_iid()
        links = db.list_links(my_iid)

        for l in links:
            if l['iid'] == iid:
                return l['public']

        return False


    def process_group_links_list(self, db, iid, my_iid=''):
        if not my_iid:
            my_iid = iid

        v = {'items':{}, 'index': []}
        links = db.list_links(iid, type='group')

        # if not looking at own profile
        my_links = []
        if not iid == my_iid:
            my_links = db.list_links(my_iid, type='group', field='iid')


        # process links into data struct used by view
        for l in links:
            liid = l['iid']
            v['index'].append(liid)
            v['items'][liid] = {}
            v['items'][liid]['iid'] = liid
            v['items'][liid]['stable'] = db.get(liid, '/info/modified')
            v['items'][liid]['link'] = l
            v['items'][liid]['visibility'] = l['public']
            v['items'][liid]['name'] = db.get_profile_name(liid)
            if not v['items'][liid]['name']:
                v['items'][liid]['name'] = l['title']

            v['items'][liid]['picture'] = db.get_profile_picture(liid)
            if not v['items'][liid]['picture']:
                v['items'][liid]['picture'] = '/static/dummy_group.png'
            else:
                # convert to full URL
                v['items'][liid]['picture'] = 'http://%s.hexlabel.net/get%s' % (liid, v['items'][liid]['picture'])

            if liid in my_links:
                v['items'][liid]['is_mutual'] = True
            else:
                v['items'][liid]['is_mutual'] = False

            if iid in db.list_members(liid):
                v['items'][liid]['is_member'] = True
                info = db.get_json(liid, '/auth/' + my_iid)
                if info:
                    my_roles = info.get('roles', [])
                    if 'admin' in my_roles:
                        v['items'][liid]['is_admin'] = True
                    else:
                        v['items'][liid]['is_admin'] = False
                else:
                    v['items'][liid]['is_admin'] = False
            else:
                v['items'][liid]['is_member'] = False
                v['items'][liid]['is_admin'] = False

        # sort by name
        v['index'].sort(key=lambda x: v['items'][x]['name'].lower())

        return v


    def process_member_list(self, db, iid, my_iid):
        v = {'items':{}, 'index':[]}

        members = db.list_members(iid)

        my_friend_list = db.list_friends(my_iid)

        for miid in members:
            v['index'].append(miid)

            info = db.get_json(iid, '/auth/' + miid)
            roles = info.get('roles', [])

            v['items'][miid] = {}
            v['items'][miid]['iid'] = miid
            v['items'][miid]['stable'] = db.get(miid, '/info/modified')
            v['items'][miid]['roles'] = roles
            v['items'][miid]['name'] = db.get_profile_name(miid)
            if not v['items'][miid]['name']:
                v['items'][miid]['name'] = db.get_json(iid, '/auth/'+miid)
                if v['items'][miid]['name'] and 'name' in v['items'][miid]['name']:
                    v['items'][miid]['name'] = v['items'][miid]['name']['name']
                else:
                    v['items'][miid]['name'] = _('Unknown Person')

            v['items'][miid]['picture'] = db.get_profile_picture(miid)
            if not v['items'][miid]['picture']:
                v['items'][miid]['picture'] = '/static/dummy_group.png'
            else:
                # convert to full URL
                v['items'][miid]['picture'] = 'http://%s.hexlabel.net/get%s' % (miid, v['items'][miid]['picture'])

            if miid == my_iid:
                v['items'][miid]['is_self'] = True
            else:
                v['items'][miid]['is_self'] = False

            if miid in my_friend_list:
                v['items'][miid]['is_friend'] = True
            else:
                v['items'][miid]['is_friend'] = False

        # sort by name
        v['index'].sort(key=lambda x: v['items'][x]['name'].lower())

        return v


    def process_person_profile(self, db, iid):
        # process fields for display
        v = {}

        v['iid'] = iid
        v['meta'] = {}
        v['meta']['last_updated'] = ''
        e = db.get_last_event(iid)
        if e:
            v['meta']['last_updated'] = utils.utcisostrlocal(e['timestamp'])
            v['meta']['pretty_last_updated'] = utils.prettydate(e['timestamp'])

        v['generic'] = {}
        #-- name (always public)
        v['generic']['name'] = {'meta':{}, 'values':[{}]}
        v['generic']['name']['meta']['label'] = 'Name'
        v['generic']['name']['values'][0]['id'] = 'generic:0:name'
        v['generic']['name']['values'][0]['value'] = db.get_profile_name(iid)
        if not v['generic']['name']['values'][0]['value']:
            _('Unkown Person')

        v['generic']['name']['values'][0]['error'] = None
        v['generic']['name']['values'][0]['visibility'] = True
        
        #-- profile picture (always public)
        v['generic']['picture'] = {'meta':{}, 'values':[{}], 'is_dummy': False}
        v['generic']['picture']['meta']['label'] = 'Picture'
        v['generic']['picture']['values'][0]['id'] = 'generic:0:picture'
        v['generic']['picture']['values'][0]['value'] = db.get_profile_picture(iid)
        v['generic']['picture']['values'][0]['error'] = None
        v['generic']['picture']['values'][0]['visibility'] = True
        if not v['generic']['picture']['values'][0]['value']:
            v['generic']['picture']['values'][0]['value'] = '/static/dummy_person.png'
            v['generic']['picture']['is_dummy'] = True
        else:
            # convert to full URL
            v['generic']['picture']['values'][0]['value'] = 'http://%s.hexlabel.net/get%s' % (iid, v['generic']['picture']['values'][0]['value'])
        
        #-- about me blurb
        v['generic']['about'] = {'meta':{}, 'values':[{}]}
        v['generic']['about']['meta']['label'] = 'About'
        v['generic']['about']['values'][0]['id'] = 'generic:0:about'
        v['generic']['about']['values'][0]['error'] = None

        value, visibility = db.get_profile_text(iid)
        # [NOTE: hack for legacy bug]
        if value == 'None':
            value = ''

        v['generic']['about']['values'][0]['value'] = value
        v['generic']['about']['values'][0]['visibility'] = visibility
        
        #-- other
        info = db.get_personal_info(iid)
        for section in info.list_section_names():
            v[section] = {}
            v[section]['meta'] = {}
            num_vis_sec = 0
            num_set_sec = 0
            for field in info.list_field_names(section):
                v[section][field] = {'meta':{}, 'values':[]}
                v[section][field]['meta']['label'] = field.capitalize()

                i = 0
                num_vis_fld = 0
                num_set_fld = 0
                for value, visibility in info.list_field_values(section, field):
                    v[section][field]['values'].append({})
                    v[section][field]['values'][i]['id'] = '%s:%s:%s' % (section, i, field)
                    v[section][field]['values'][i]['value'] = value
                    v[section][field]['values'][i]['visibility'] = visibility

                    if value:
                        num_set_fld += 1
                        if visibility:
                            num_vis_fld += 1

                    v[section][field]['values'][i]['error'] = None
                    i += 1

                v[section][field]['meta']['num_visible'] = num_vis_fld
                v[section][field]['meta']['num_set'] = num_set_fld
                if num_set_fld > 0:
                    num_set_sec += 1

                if num_vis_fld > 0:
                    num_vis_sec += 1

            v[section]['meta']['num_visible'] = num_vis_sec
            v[section]['meta']['num_set'] = num_set_sec

        # fill in blanks for the basic fields
        basic = (('personal', ('born', 'gender')), ('location', ('country', 'region', 'city', 'lang')), ('contact', ('email', 'website', 'phone', 'im')), ('other', ('status','status_timestamp', 'pretty_status_timestamp')))
        for basic_section, basic_fields in basic:
            if not v.get(basic_section):
                v[basic_section] = {}                        
                v[basic_section]['meta'] = {}                        
                v[basic_section]['meta']['num_visible'] = 0
                v[basic_section]['meta']['num_set'] = 0

            for basic_field in basic_fields:
                if not v[basic_section].get(basic_field):
                    v[basic_section][basic_field] = {'meta':{}, 'values':[{}]}
                    v[basic_section][basic_field]['meta']['label'] = basic_field.capitalize()
                    v[basic_section][basic_field]['meta']['num_visible'] = 0
                    v[basic_section][basic_field]['meta']['num_set'] = 0
                    v[basic_section][basic_field]['values'][0]['id'] = '%s:0:%s' % (basic_section, basic_field)
                    v[basic_section][basic_field]['values'][0]['value'] = ''
                    v[basic_section][basic_field]['values'][0]['visibility'] = False
                    v[basic_section][basic_field]['values'][0]['error'] = None
        
        # adjust some labels
        v['generic']['about']['meta']['label'] = _('About me')
        v['location']['lang']['meta']['label'] = _('Language')
        v['contact']['im']['meta']['label'] = _('IM')

        # prettify status timestamp if exists
        v['other']['pretty_status_timestamp']['values'][0]['value'] = utils.prettydate(v['other']['status_timestamp']['values'][0]['value']) 

        return v


    def process_group_profile(self, db, iid):
        # process fields for display
        v = {}

        v['iid'] = iid
        v['meta'] = {}
        v['meta']['last_updated'] = ''
        if iid:
            e = db.get_last_event(iid)
            if e:
                v['meta']['last_updated'] = utils.utcisostrlocal(e['timestamp'])
                v['meta']['pretty_last_updated'] = utils.prettydate(e['timestamp'])

        v['generic'] = {}

        #-- name (always public)
        v['generic']['name'] = {'meta':{}, 'values':[{}]}
        v['generic']['name']['meta']['label'] = 'Name'
        v['generic']['name']['values'][0]['id'] = 'generic:0:name'
        if iid:
            v['generic']['name']['values'][0]['value'] = db.get_profile_name(iid) or _('Unkown Group')
        else:
            v['generic']['name']['values'][0]['value'] = ''

        v['generic']['name']['values'][0]['error'] = None
        v['generic']['name']['values'][0]['visibility'] = True

        if not v['generic']['name']['values'][0]['value']:
            v['generic']['name']['values'][0]['value'] = 'Unnamed Group'
            
    
        #-- profile picture (always public)
        v['generic']['picture'] = {'meta':{}, 'values':[{}], 'is_dummy': False}
        v['generic']['picture']['meta']['label'] = 'Picture'
        v['generic']['picture']['values'][0]['id'] = 'generic:0:picture'
        if iid:
            v['generic']['picture']['values'][0]['value'] = db.get_profile_picture(iid)
        else:
            v['generic']['picture']['values'][0]['value'] = ''

        v['generic']['picture']['values'][0]['error'] = None
        v['generic']['picture']['values'][0]['visibility'] = True
        if not v['generic']['picture']['values'][0]['value']:
            v['generic']['picture']['values'][0]['value'] = '/static/dummy_group.png'
            v['generic']['picture']['is_dummy'] = True
        
        # convert to full URL
        elif iid:
            v['generic']['picture']['values'][0]['value'] = 'http://%s.hexlabel.net/get%s' % (iid, v['generic']['picture']['values'][0]['value'])
        
        #-- about me blurb
        v['generic']['about'] = {'meta':{}, 'values':[{}]}
        v['generic']['about']['meta']['label'] = 'About'
        v['generic']['about']['values'][0]['id'] = 'generic:0:about'
        v['generic']['about']['values'][0]['error'] = None

        if iid:
            value, visibility = db.get_profile_text(iid)
        else:
            value, visibility = ('', False)

        v['generic']['about']['values'][0]['value'] = value
        v['generic']['about']['values'][0]['visibility'] = visibility

        return v


    def process_application_profile(self, db, iid):
        v = {}

        v['iid'] = iid
        v['meta'] = {}
        v['meta']['last_updated'] = ''
        e = db.get_last_event(iid)
        if e:
            v['meta']['last_updated'] = utils.utcisostrlocal(e['timestamp'])
            v['meta']['pretty_last_updated'] = utils.prettydate(e['timestamp'])

        v['name'] = db.get(iid, '/info/title.txt') or _('Unknown Application')
        v['system'] = db.get_json(iid, '/application/prefs/system.json')
        v['profile'] = db.get_json(iid, '/info/application.json')
        v['picture'] = db.get_profile_picture(iid)

        if not v['picture']:
            v['picture'] = '/static/dummy_application.png'
        else:
            # convert to full URL
            v['picture'] = 'http://%s.hexlabel.net/get%s' % (iid, v['picture'])

        # get the author's profile
        if v['profile']:
            v['author_profile'] = self.process_person_profile(db, db.get_dataset_creator(iid))
        else:
            v['author_profile'] = None

        # get the content
        v['content'] = self.process_any_application_content_list(db, iid)


        system = db.get_json(iid, '/application/prefs/system.json')
        if system:
            v['system'] = system
        else:
            v['system'] = {}
            v['system']['version'] = 'Unknown'
            v['system']['principal_aware'] = False
            v['system']['group_aware'] = False

        return v

    def process_any_application_content_list(self, db, appiid):
        v = []

        # try to find the content datasets which are instances of appiid
        all_content = db.list_datasets('content')
        for ciid in all_content:
            if db.get(ciid, '/info/appiid') == appiid:
                l = {}
                l['iid'] = ciid
                l['title'] = db.get_profile_name(l['iid']) or _('Unknown Content')
                l['stable'] = db.get(l['iid'], '/info/modified')
                l['picture'] = db.get_profile_picture(l['iid'])
                if not l['picture']:
                    l['picture'] = '/static/dummy_content.png'
                else:
                    l['picture'] = 'http://%s.hexlabel.net/get%s' % (l['iid'], l['picture'])

                l['parent'] = {}
                l['parent']['iid'] = db.get(l['iid'], '/info/parent')
                if l['parent']['iid']:
                    l['parent']['type'] = db.get_dataset_type(l['parent']['iid'])
                    l['parent']['name'] = db.get_profile_name(l['parent']['iid']) or _('Unknown')
                    l['parent']['stable'] = db.get(l['parent']['iid'], '/info/modified')
                    l['parent']['picture'] = db.get_profile_picture(l['parent']['iid'])
                    if not l['parent']['picture']:
                        if l['parent']['type'] == 'group':
                            l['parent']['picture'] = '/static/dummy_group.png'
                        else:
                            l['parent']['picture'] = '/static/dummy_person.png'
                    else:
                        l['parent']['picture'] = 'http://%s.hexlabel.net/get%s' % (l['parent']['iid'], l['parent']['picture'])
                v.append(l)
        return v
        

    def process_user_application_content_list(self, db, appiid):
        v = []
        my_iid = db.get_my_iid()

        links = db.list_links(my_iid)
        for l in links:
            if db.get_dataset_type(l['iid']) == 'content':
                if db.get(l['iid'], '/info/appiid') == appiid:
                    l['title'] = db.get_profile_name(l['iid']) or l['title']
                    l['stable'] = db.get(l['iid'], '/info/modified')
                    l['picture'] = db.get_profile_picture(l['iid'])
                    if not l['picture']:
                        l['picture'] = '/static/dummy_content.png'
                    else:
                        l['picture'] = 'http://%s.hexlabel.net/get%s' % (l['iid'], l['picture'])
                    v.append(l)

            elif db.get_dataset_type(l['iid']) == 'group':
                glinks = db.list_links(l['iid'])
                for gl in glinks:
                    if db.get_dataset_type(gl['iid']) == 'content':
                        if db.get(gl['iid'], '/info/appiid') == appiid:
                            l['title'] = db.get_profile_name(gl['iid']) or l['title']
                            gl['stable'] = db.get(gl['iid'], '/info/modified')
                            gl['picture'] = db.get_profile_picture(gl['iid'])
                            if not gl['picture']:
                                gl['picture'] = '/static/dummy_content.png'
                            else:
                                gl['picture'] = 'http://%s.hexlabel.net/get%s' % (gl['iid'], gl['picture'])

                            v.append(gl)
        return v


    def process_content_list(self, db, iid, all_content=False):
        v = {}
        v['items'] = {}
        v['index'] = []
        v['person_index'] = []
        v['group_index'] = []
        v['app_index'] = {}
        v['app_meta'] = {}

        links = db.list_links(iid)
        for l in links:
            if l['type'] == 'content':
                ciid = l['iid']
                v['index'].append(ciid)
                v['person_index'].append(ciid)
                v['items'][ciid] = {}
                v['items'][ciid]['iid'] = l['iid']
                v['items'][ciid]['stable'] = db.get(ciid, '/info/modified')
                v['items'][ciid]['title'] = db.get_profile_name(l['iid']) or l['title']
                v['items'][ciid]['picture'] = db.get_profile_picture(l['iid'])
                v['items'][ciid]['visibility'] = l['public']
                v['items'][ciid]['update'] = False
                v['items'][ciid]['system'] = db.get_json(ciid, '/application/prefs/system.json')

                appiid = db.get(ciid, '/info/appiid')
                v['items'][ciid]['application'] = {}
                v['items'][ciid]['application']['iid'] = appiid
                if v['items'][ciid]['application']['iid']:
                    v['items'][ciid]['application']['name'] = db.get_profile_name(appiid) or _('Unknown Application')

                    # get the application profile
                    v['items'][ciid]['application'] = self.process_application_profile(db, appiid)

                    # determine if an update is possible
                    if v['items'][ciid]['system']:
                        if v['items'][ciid]['system']['version'] < v['items'][ciid]['application']['system']['version']:
                            v['items'][ciid]['update'] = True

                else:
                    v['items'][ciid]['application']['name'] = _('Unknown Application')
                    v['items'][ciid]['application']['profile'] = None
                    v['items'][ciid]['application']['author_profile'] = None
                    v['items'][ciid]['application']['system'] = { 'version': 1.0 }

                if not v['app_index'].get(appiid):
                    v['app_index'][appiid] = []
                    v['app_meta'][appiid] = {}
                    v['app_meta'][appiid]['iid'] = appiid
                    v['app_meta'][appiid]['name'] = v['items'][ciid]['application']['name']
                    v['app_meta'][appiid]['profile'] = v['items'][ciid]['application']['profile']
                    v['app_meta'][appiid]['author_profile'] = v['items'][ciid]['application']['author_profile']
                    v['app_meta'][appiid]['version'] = v['items'][ciid]['application']['system']['version']
                    v['app_meta'][appiid]['update'] = v['items'][ciid]['update']
                else:
                    if not v['app_meta'][appiid]['update']:
                        v['app_meta'][appiid]['update'] = v['items'][ciid]['update']

                v['app_index'][appiid].append(ciid)


                v['items'][ciid]['parent'] = {}
                v['items'][ciid]['parent']['iid'] = iid
                if v['items'][ciid]['parent']['iid']:
                    v['items'][ciid]['parent']['name'] = db.get_profile_name(iid)
                    if not v['items'][ciid]['parent']['name']:
                        v['items'][ciid]['parent']['name'] = _('Unknown')
                else:
                    v['items'][ciid]['parent']['name'] = None
                    

                # assume that are only linking to own content
                v['items'][ciid]['parent']['is_admin'] = True

                v['items'][ciid]['link'] = l
                if not v['items'][ciid]['picture']:
                    v['items'][ciid]['picture'] = '/static/dummy_content.png'
                else:
                    v['items'][ciid]['picture'] = 'http://%s.hexlabel.net/get%s' % (l['iid'], v['items'][ciid]['picture'])

        # try to find other application datasets which do not have content instances
        all_apps = db.list_datasets('application')
        for appiid in all_apps:
            if not v['app_index'].get(appiid):
                appname = db.get_profile_name(appiid) or _('Unkown Application')
                v['app_index'][appiid] = []
                v['app_meta'][appiid] = {}
                v['app_meta'][appiid]['iid'] = appiid
                v['app_meta'][appiid]['name'] = appname
                v['app_meta'][appiid]['update'] = False
                v['app_meta'][appiid]['version'] = _('Uknown')

                # get the author's profile
                v['app_meta'][appiid]['profile'] = db.get_json(appiid, '/info/application.json')
                if v['app_meta'][appiid]['profile']:
                    v['app_meta'][appiid]['author_profile'] = self.process_person_profile(db, v['app_meta'][appiid]['profile']['author'])
                else:
                    v['app_meta'][appiid]['author_profile'] = None

        # sort by application, name
        # FIXME: review this; do we need all these indexes? do we need to sort them by two keys?
        v['index'].sort(cmp=lambda x, y: cmp(v['items'][x]['application']['iid'], v['items'][y]['application']['iid']) or cmp(v['items'][x]['title'].lower(), v['items'][y]['title'].lower()))

        v['person_index'].sort(cmp=lambda x, y: cmp(v['items'][x]['application']['iid'], v['items'][y]['application']['iid']) or cmp(v['items'][x]['title'].lower(), v['items'][y]['title'].lower()))

        v['group_index'].sort(cmp=lambda x, y: cmp(v['items'][x]['application']['iid'], v['items'][y]['application']['iid']) or cmp(v['items'][x]['title'].lower(), v['items'][y]['title'].lower()))

        for iid, index in v['app_index'].items():
            index.sort(cmp=lambda x, y: cmp(v['items'][x]['application']['iid'], v['items'][y]['application']['iid']) or cmp(v['items'][x]['title'].lower(), v['items'][y]['title'].lower()))

        return v

    def process_content_info(self, db, iid):
        v = {}
        v['peerscape_version'] = VERSION
        v['iid'] = iid
        v['hexlabel'] = iid
        v['title'] = db.get(v['hexlabel'], '/info/title.txt')
        v['modified'] = db.get(v['hexlabel'], '/info/modified')
        v['created'] = db.get(v['hexlabel'], '/info/created')
        v['picture'] = db.get_profile_picture(v['hexlabel'])

        v['owner'] = {}
        v['owner']['iid'] = db.get_dataset_creator(v['hexlabel'])
        v['owner']['name'] = db.get_profile_name(v['owner']['iid'])

        v['viewer'] = {}
        v['viewer']['iid'] = db.get_my_iid()
        v['viewer']['name'] = db.get_profile_name(v['viewer']['iid'])
        v['viewer']['group_member'] = 'false'
        v['viewer']['group_roles'] = '[]'

        v['parent'] = {}
        v['parent']['iid'] = db.get(v['hexlabel'], '/info/parent')
        if v['parent']['iid']:
            v['parent']['name'] = db.get_profile_name(v['parent']['iid'])
            v['parent']['type'] = db.get_dataset_type(v['parent']['iid'])
            if v['parent']['type'] == 'group':
                if v['owner']['iid'] in db.list_members(v['parent']['iid']):
                    v['viewer']['group_member'] = 'true'
                else:
                    v['viewer']['group_member'] = 'false'

                info = db.get_json(v['parent']['iid'], '/auth/' + v['viewer']['iid'])
                if info and info.has_key('roles'):
                    v['viewer']['group_roles'] = json.dumps(info['roles'])
                else:
                    v['viewer']['group_roles'] = json.dumps([])
            else:
                v['viewer']['group_member'] = 'false'
                v['viewer']['group_roles'] = json.dumps([])
        else:
            v['parent']['name'] = None
            v['parent']['type'] = None

        v['app'] = {}
        v['app']['iid'] = db.get(v['hexlabel'], '/info/appiid')
        if v['app']['iid']:
            v['app']['name'] = db.get(v['app']['iid'], '/info/title.txt')
        else:
            v['app']['name'] = None

        return v



    # INVITATIONS
    # ---------------
    def get_invitations(self, db, force_refresh=False):
        if not self._invitations_awake or force_refresh:
            self._expand_invitations_rep(db)

        return self._transient_invitations

    
    def get_invitation(self, db, iid, force_refresh=False):
        if not self._invitations_awake or force_refresh:
            self._expand_invitations_rep(db)

        return self._transient_invitations[iid]

    
    def is_invitation_pending(self, db, iid, inviter=None):
        return db.is_invitation_pending(iid, inviter)
        

    def add_invitation(self, db, real_invitation, code=None):
        if not self._invitations_awake:
            self._expand_invitations_rep(db)

        real_invitation = real_invitation.decode()
        if real_invitation:
            self._transient_invitations[real_invitation.iid] = self._expand_invitation(db, real_invitation, code)


    def accept_invitation(self, db, iid):
        if not self._invitations_awake:
            self._expand_invitations_rep(db)

        return self._transient_invitations[iid]['invitation'].accept(db)        


    def delete_invitation(self, db, iid):
        if not self._invitations_awake:
            self._expand_invitations_rep(db)

        db.clear_pending_invitation(iid)
        del self._transient_invitations[iid]


    def _expand_invitations_rep(self, db):
        rep = db.list_pending_invitations()

        for item in rep:
            i_iid = item['iid']
            if not self._transient_invitations.has_key(i_iid):
                real_invitation = item['invitation']
                if real_invitation:
                    self._transient_invitations[i_iid] = self._expand_invitation(db, real_invitation, item.get('code'))


    def _expand_invitation(self, db, real_invitation, code):
        stable = db.get(real_invitation.iid, '/info/modified')
        type = db.get_dataset_type(real_invitation.iid)
        name = db.get_profile_name(real_invitation.iid)
        if stable:
            if not name:
                if type == 'group':
                    name = _('Unknown Group')
                else:
                    name = _('Unknown Person')

            picture = db.get_profile_picture(real_invitation.iid)
            if not picture: 
                if type == 'group':
                    picture = '/static/dummy_group.png'
                else:
                    picture = '/static/dummy_person.png'
            else:
                picture = 'http://%s.hexlabel.net/get%s' % (real_invitation.iid, picture)
        else:
            name = 'Some person or group'
            picture = '/static/dummy_unknown.png'

        return {'iid': real_invitation.iid, 'stable': stable, 'code': code, 'invitation':real_invitation, 'name': name, 'picture': picture, 'type': type}

